﻿package dut.ir.lxb.weibo.spider;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;

import dut.ir.lxb.weibo.bean.EventBean;
import dut.ir.lxb.weibo.bean.UserBean;
import dut.ir.lxb.weibo.bean.UserWeiboAction;
import dut.ir.lxb.weibo.bean.WeiboBean;
import dut.ir.lxb.weibo.db.UserWeiboActionDao;
import dut.ir.lxb.weibo.db.WeiboInfoDao;
import dut.ir.lxb.weibo.util.ActionType;
import dut.ir.lxb.weibo.util.RelationType;
import dut.ir.lxb.weibo.util.SpiderConfig;
import dut.ir.lxb.weibo.util.SpiderUtility;

public class WeiboDownloader
{
	
	protected LoginUser loginUser;
	
	protected HttpClient loginClient;
	
	public static WeiboParser weiboParser;
	
	protected boolean isLogin = false;
	
	protected boolean isAccessDataBase = false;
	
	static
	{
		weiboParser = new WeiboParser();
	}
	
	
	
	public LoginUser getLoginUser()
	{
	
		return loginUser;
	}

	public void setLoginUser(LoginUser loginUser)
	{
	
		this.loginUser = loginUser;
	}

	public WeiboDownloader()
	{
		loginUser = new LoginUser();
	}
	
	public WeiboDownloader(String userName, String password)
	{
		loginUser = new LoginUser(userName, password);
	}
	
	public WeiboDownloader(LoginUser loginUser)
	{
		this.loginUser = loginUser;
	}
	
	public boolean isAccessDataBase()
	{
	
		return isAccessDataBase;
	}

	public void setAccessDataBase(boolean isAccessDataBase)
	{
	
		this.isAccessDataBase = isAccessDataBase;
	}
	
	

	public boolean isLogin()
	{
	
		return isLogin;
	}

	public void setLogin(boolean isLogin)
	{
	
		this.isLogin = isLogin;
	}

	/**
	 * 初始化
	 */
	public void init()
	{
		loginClient = loginUser.login();
		if(loginClient != null)
			isLogin = true;
		else
			isLogin = false;
	}
	
	public void finalise()
	{
		if(loginUser.logout(loginClient))
		{
			System.out.println(loginUser.getUserName() + "登出成功。");
			isLogin = false;
		}
	}
	
	/**
	 * 通过微博URL下载微博
	 * @param url
	 * @return
	 */
	public WeiboBean downloadWeiboByUrl(String url) throws AccountForbiddonException
	{
		if(!isLogin)
			init();
		if(!isLogin)
		{
			System.out.println("对不起，登录失败。");
			return null;
		}
		
		String uid = url.substring("http://weibo.com/".length(), url.lastIndexOf('/'));
		UserBean ub = downloadUserInfo(uid);
		
		GetMethod getMethod = new GetMethod(url);
		try
		{
			loginClient.executeMethod(getMethod);
			if(getMethod.getStatusCode() != HttpStatus.SC_OK)
				return null;
			InputStream is = getMethod.getResponseBodyAsStream();
			BufferedReader br = new BufferedReader(new InputStreamReader(is, SpiderConfig.getWeiboEncoding()));
			StringBuilder sb = new StringBuilder();
			String currentLine = br.readLine();
			while(currentLine != null)
			{
				sb.append(currentLine);
				sb.append("\n");
				currentLine = br.readLine();
			}
			
			String respStr = sb.toString();
			return weiboParser.parseWeiboByUrlPage(respStr, ub);
		}
		catch (HttpException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
			return null;
		}
		catch (IOException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
			return null;
		}
		catch(Exception e)
		{
			e.printStackTrace();
			return null;
		}
		finally
		{
			getMethod.releaseConnection();
		}
		
		
	}
	
	/**
	 * 下载用户微博行为，评论转发点赞等
	 * @param mid
	 * @param actionType
	 * @return
	 */
	public List<UserWeiboAction> downloadUserWeiboActionGivenPageCount(WeiboBean wb, ActionType actionType, int pageCount) throws AccountForbiddonException
	{
		//先验证登录
		if(!isLogin)
			init();
		if(!isLogin)
			return null;
		List<UserWeiboAction> uwActionList = new ArrayList<UserWeiboAction>();
		String templeUrl = "";
		if(actionType == ActionType.COMMENT)
			templeUrl = SpiderConfig.getCommentUrl();
		else if(actionType == ActionType.FORWARD)
			templeUrl = SpiderConfig.getForwardUrl();
		else if(actionType == ActionType.LIKE)
			templeUrl = SpiderConfig.getLikeUrl();
		else
		{
			System.out.println("URL格式错误！");
			return null;
		}
		
		String mid = wb.getMid();
		//请求数据URL
		String requestUrl = String.format(templeUrl, wb.getMid(), 1);
		GetMethod getMethod = new GetMethod(requestUrl);
		try
		{
			for(int i = 1; i <= pageCount; i++)
			{
				
				
				requestUrl = String.format(templeUrl, mid, i);
				getMethod = new GetMethod(requestUrl);
				loginClient.executeMethod(getMethod);
				Thread.currentThread();
				Thread.sleep(SpiderConfig.getWaitTime() / 2);
				String respStr = SpiderUtility.streamToString(getMethod.getResponseBodyAsStream(), SpiderConfig.getWeiboEncoding());//new String(getMethod.getResponseBody(), SpiderConfig.getWeiboEncoding());
				getMethod.releaseConnection();
				
				//检查是否被封号，若被封则抛出异常
				if(actionType != ActionType.LIKE && !SpiderUtility.checkJsonResponse(respStr))
					throw new AccountForbiddonException("帐号被封，现在切换帐号");

				
				List<UserWeiboAction> curPageLst = weiboParser.parseUserWeiboActionList(respStr, wb, actionType);
				uwActionList.addAll(curPageLst);
				SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
				String dateValue = df.format(new Date());
				String forwardInfo = String.format("[%s]已下载用户 %s[%s] 微博 [%s] 第[%d/%d]页的%d条%s信息", dateValue, wb.getNickname(), wb.getUid(), wb.getMid(), i, pageCount, curPageLst.size(), actionType.toString());
				System.out.println(forwardInfo);
				//////
				if(isAccessDataBase)
				{
					//是否访问数据库，若访问，则插入数据，且清空
					UserWeiboActionDao actionDao = UserWeiboActionDao.getInstance();
					actionDao.insertActionList(uwActionList);
					curPageLst.clear();
					curPageLst = null;
					uwActionList.clear();
						
				}
				
			}
			
			
		} 
		catch (HttpException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		} 
		catch (IOException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		} 
		catch (InterruptedException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		catch (Exception e)
		{
			e.printStackTrace();
			if(e instanceof AccountForbiddonException)
				throw new AccountForbiddonException("账号被封！");
		}
		finally
		{
			if(!getMethod.isAborted())
				getMethod.abort();
		}
		
		return uwActionList;
	}
	
	
	/**
	 * 下载用户微博行为，评论转发点赞等
	 * @param mid
	 * @param actionType
	 * @return
	 */
	public List<UserWeiboAction> downloadUserWeiboAction(WeiboBean wb, ActionType actionType) throws AccountForbiddonException
	{
		//先验证登录
		if(!isLogin)
			init();
		if(!isLogin)
			return null;
		String templeUrl = "";
		
		int downloadedCount = UserWeiboActionDao.getInstance().getUserActionCountByMid(wb.getMid(), actionType);
		int downloadedPage = 0;
		int actionCount = 0;
		
		if(actionType == ActionType.COMMENT)
		{
			templeUrl = SpiderConfig.getCommentUrl();
			downloadedPage = downloadedCount / SpiderConfig.getCommentCountPerpage();
			actionCount = wb.getCommentNum();
		}
		else if(actionType == ActionType.FORWARD)
		{
			templeUrl = SpiderConfig.getForwardUrl();
			downloadedPage = downloadedCount / SpiderConfig.getForwardCountPerpage();
			actionCount = wb.getForwardNum();
		}
		else if(actionType == ActionType.LIKE)
		{
			templeUrl = SpiderConfig.getLikeUrl();
			downloadedPage = downloadedCount / SpiderConfig.getLikeCountPerpage();
			actionCount = wb.getLikeNum();
		}
		else
		{
			System.out.println("URL格式错误！");
			return null;
		}
		
		if(actionCount == 0) return null;
		//请求数据URL
		String requestUrl = String.format(templeUrl, wb.getMid(), 1);
		GetMethod getMethod = new GetMethod(requestUrl);
		try
		{
			loginClient.executeMethod(getMethod);
			Thread.currentThread();
			Thread.sleep(SpiderConfig.getWaitTime());
			String respStr = SpiderUtility.streamToString(getMethod.getResponseBodyAsStream(), SpiderConfig.getWeiboEncoding());//new String(getMethod.getResponseBody(), SpiderConfig.getWeiboEncoding());
			getMethod.releaseConnection();
			
			int pageCount = weiboParser.getPageCount(respStr);
			int needDownloadPage = pageCount - downloadedPage;
			
			System.out.println(String.format("用户[%s]的微博[%s][%s]信息共有[%d]页，已下载[%d]页，下载剩余的[%d]页", wb.getNickname(), wb.getMid(), actionType, pageCount, downloadedPage, needDownloadPage));
			if(needDownloadPage <= 0) return null;
			return downloadUserWeiboActionGivenPageCount(wb, actionType, needDownloadPage);
			
		} 
		catch (HttpException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		} 
		catch (IOException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		} 
		catch (InterruptedException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		catch (Exception e)
		{
			e.printStackTrace();
			if(e instanceof AccountForbiddonException)
				throw new AccountForbiddonException("账号被封！");
		}
		finally
		{
			if(!getMethod.isAborted())
				getMethod.abort();
		}
		
		return null;
	}
	
	/**
	 * 下载用户关注列表
	 * @param uid
	 * @return
	 */
	public List<String> downloadUserFollowList(UserBean ub, RelationType relationType)
	{
		if(!isLogin)
			return null;
		List<String> followList = null;
		
		if(ub == null || ub.getFollow() < 0)
			return null;
		if(!isLogin)
			init();
		int pageCount = ub.getFollow() / SpiderConfig.getMaxFollowPerpage();
		if(ub.getFollow() % SpiderConfig.getMaxFollowPerpage() != 0)
			pageCount++;
		if(pageCount > (SpiderConfig.getMaxFollowCount() / SpiderConfig.getMaxFollowPerpage()))
			pageCount = SpiderConfig.getMaxFollowCount() / SpiderConfig.getMaxFollowPerpage();
		
		String templeUrl = "";
		if(relationType == RelationType.FANS)
			templeUrl = SpiderConfig.getFansUrl();
		else if(relationType == RelationType.FOLLOW)
			templeUrl = SpiderConfig.getFollowUrl();
		else
		{
			System.out.println("URL格式错误");
			return null;
		}
		
		
		try
		{
			String requestUrl = "";
			for(int i = 1; i <= pageCount; i++)
			{
				requestUrl = String.format(templeUrl, ub.getUid(), 1);
				GetMethod getMethod = new GetMethod(requestUrl);
				loginClient.executeMethod(getMethod);
				try
				{
					Thread.currentThread();
					Thread.sleep(1000);
				} catch (InterruptedException e)
				{
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
				String respStr = new String(getMethod.getResponseBody(), SpiderConfig.getWeiboEncoding());
				followList = weiboParser.parseFollowList(respStr);
				
				getMethod.releaseConnection();
			}
			
			
		} catch (HttpException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		catch (IOException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		
		return followList;
	}
	
	public List<WeiboBean> downloadEventHotWeiboList(EventBean eb) throws AccountForbiddonException
	{
		if(!isLogin)
		{
			//System.out.println("对不起，您还没有登录成功，无法下载微博！");
			init();
		}
		if(!isLogin)
			return null;
		
		List<WeiboBean> wbList = new ArrayList<WeiboBean>();
			
		String requestUrl = String.format("http://weibo.com/p/%s?page=%d", eb.getEventCode(), 1);
		
		//三次Get方法请求微博数据
		GetMethod method = new GetMethod(requestUrl);
		try
		{
			loginClient.executeMethod(method);
			String respStr = SpiderUtility.streamToString(method.getResponseBodyAsStream(), SpiderConfig.getEventEncoding());
			respStr = respStr.replace("", "");
			//检查帐号是否正常
//			if(!SpiderUtility.checkHtmlResponse(respStr))
//				throw new AccountForbiddonException();
			
			//String respStr = method.getResponseBodyAsString();
			wbList = weiboParser.parseEventHotWeibo(respStr);
			SpiderUtility.wrapEventName(wbList, eb);
			if(isAccessDataBase)
			{
				WeiboInfoDao weiboDao = WeiboInfoDao.getInstance();
				weiboDao.insertWeiboList(wbList);
			}
			method.releaseConnection();
			
			
			Thread.currentThread();
			Thread.sleep(SpiderConfig.getWaitTime());
			
			
			
		}
		catch (HttpException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
			return null;
		} 
		catch (IOException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
			return null;
		} 
		catch (InterruptedException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		finally
		{
			if(!method.isAborted())
				method.abort();
		}
		
		
		
		
		return wbList;
	}
	
	/**
	 * To be Continued
	 * @param eb
	 * @return
	 */
	public List<WeiboBean> downloadEventWeiboList(EventBean eb) throws AccountForbiddonException
	{
		if(!isLogin)
		{
			//System.out.println("对不起，您还没有登录成功，无法下载微博！");
			init();
		}

		if(!isLogin)
			return null;
		
		String [] requestUrls = new String[3];
		int pageCount = 1, maxPage = 2;
		String eventCode = eb.getEventCode();
		boolean isMaxPage = false;
		List<WeiboBean> wbList = new ArrayList<WeiboBean>();
		String plName = SpiderConfig.getDomid();
		while(pageCount <= maxPage)
		{
			int currentPage = 3 * (pageCount - 1);
			int page = pageCount;
			long time = System.currentTimeMillis();
			requestUrls[0] = String.format("http://weibo.com/p/%s?&page=%d#Pl_Third_App__10", eb.getEventCode(), page);
			requestUrls[1] = 
					String.format("http://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain=100808&from=faxian_huati&pagebar=0&tab=home&current_page=%d&pl_name=%s&id=%s&script_uri=/p/%s&feed_type=1&page=%d&pre_page=%d&domain_op=100808&__rnd=%d", currentPage + 1, plName, eventCode, eventCode, page, page, time);
					//String.format("http://weibo.com/p/aj/v6/mblog/mbloglist?from=faxian_huati&ajwvr=6&domain=100808&current_page=%d&page=%d&since_id=2399999970&pagebar=0&tab=talk_home&pl_name=%s&id=%s&script_uri=/p/%s/talk_home&feed_type=1&pre_page=%d&domain_op=100808&__rnd=%d", currentPage + 1, page, plName, eb.getEventCode(), eb.getEventCode(), page, new Date().getTime());
					//String.format("http://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain=100808&since_id=2399999953&current_page=%d&page=%d&pagebar=0&tab=home&pl_name=%s&id=%s&script_uri=/p/%s&feed_type=1&pre_page=%d&domain_op=100808", currentPage + 1, page, plName, eb.getEventCode(), eb.getEventCode(), page);
			requestUrls[2] = 
					String.format("http://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain=100808&from=faxian_huati&pagebar=1&tab=home&current_page=%d&pl_name=%s&id=%s&script_uri=/p/%s&feed_type=1&page=%d&pre_page=%d&domain_op=100808&__rnd=%d", currentPage + 2, plName, eventCode, eventCode, page, page, time);

					//String.format("http://weibo.com/p/aj/v6/mblog/mbloglist?from=faxian_huati&ajwvr=6&domain=100808&current_page=%d&page=%d&since_id=2399999970&pagebar=1&tab=talk_home&pl_name=%s&id=%s&script_uri=/p/%s/talk_home&feed_type=1&pre_page=%d&domain_op=100808&__rnd=%d", currentPage + 2, page, plName, eb.getEventCode(), eb.getEventCode(), page, new Date().getTime());

					//String.format("http://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain=100808&since_id=2399999953&current_page=%d&page=%d&pagebar=1&tab=home&pl_name=%s&id=%s&script_uri=/p/%s&feed_type=1&pre_page=%d&domain_op=100808", currentPage + 2, page, plName, eb.getEventCode(), eb.getEventCode(), page);

			//requestUrls[2] = String.format("http://weibo.com/p/aj/v6/mblog/mbloglist?domain=100808&pre_page=%d&page=%d&count=15&pagebar=1&pl_name=Pl_Third_App__9&id=%s&feed_type=1", pageCount, pageCount, eb.getEventCode());
		
			//String.format("http://weibo.com/p/aj/v6/mblog/mbloglist?domain=100808&pre_page=%d&page=%d&count=15&pagebar=0&pl_name=Pl_Third_App__9&id=%s&feed_type=1", pageCount, pageCount, eb.getEventCode());
			
			//三次Get方法请求微博数据
			GetMethod method1 = new GetMethod(requestUrls[0]);
			GetMethod method2 = new GetMethod(requestUrls[1]);
			GetMethod method3 = new GetMethod(requestUrls[2]);
			
			try
			{
				List<WeiboBean> curPageLst = new ArrayList<WeiboBean>();
				List<WeiboBean> curReqLst = null;
				loginClient.executeMethod(method1);
				String respStr = SpiderUtility.streamToString(method1.getResponseBodyAsStream(), SpiderConfig.getEventEncoding());
				method1.releaseConnection();
				
				//检查帐号是否异常，若异常抛出异常进行处理
				if(!SpiderUtility.checkJsonResponse(respStr))
					throw new AccountForbiddonException();
				
				curReqLst = weiboParser.parseFirstEventWeibo(respStr);
				curPageLst.addAll(curReqLst);
				
				
				Thread.currentThread();
				Thread.sleep(SpiderConfig.getWaitTime());
//				if(curReqLst == null || curReqLst.size() == 0)
//					break;
				
				//比较日期，若在一个月之前直接跳过
				//if(curReqLst != null && curReqLst.size() > 0)
				//	weiboDate = df.parse(curReqLst.get(curReqLst.size() - 1).getTime());
				//if(weiboDate.before(breakPoint))
				//	break;
				
				
				
				loginClient.executeMethod(method2);
				respStr = SpiderUtility.streamToString(method2.getResponseBodyAsStream(), SpiderConfig.getEventEncoding());
				method2.releaseConnection();
				//检查帐号是否异常，若异常抛出异常进行处理
				if(!SpiderUtility.checkJsonResponse(respStr))
					throw new AccountForbiddonException();
				
				curReqLst = weiboParser.parseWeiboList(respStr, null);
				curPageLst.addAll(curReqLst);
				
				Thread.currentThread();
				Thread.sleep(SpiderConfig.getWaitTime());
//				if(curReqLst == null || curReqLst.size() == 0)
//					break;
				
				loginClient.executeMethod(method3);
				respStr = SpiderUtility.streamToString(method3.getResponseBodyAsStream(), SpiderConfig.getEventEncoding());
				method3.releaseConnection();
				//检查帐号是否异常，若异常抛出异常进行处理
				if(!SpiderUtility.checkJsonResponse(respStr))
					throw new AccountForbiddonException();
				
				curReqLst = weiboParser.parseWeiboList(respStr, null);
				curPageLst.addAll(curReqLst);
				//获得页数
				if(!isMaxPage)
				{
					maxPage = weiboParser.parseEventWeiboPageCount(respStr);
					isMaxPage = true;
				}
//				if(curReqLst == null || curReqLst.size() == 0)
//					break;
				SpiderUtility.wrapEventName(curPageLst, eb);
				wbList.addAll(curPageLst);
				
				
				SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
				String dateValue = df.format(new Date());
				String forwardInfo = String.format("[%s]已下载事件 [%s] 第[%d/%d]页的[%d]条微博信息", dateValue, eb.getEventname(), pageCount, maxPage, curPageLst.size());
				System.out.println(forwardInfo);
				
				Thread.currentThread();
				Thread.sleep(SpiderConfig.getWaitTime());
				
				//是否访问数据库，若访问，则插入数据，且清空
				if(isAccessDataBase)
				{
					WeiboInfoDao weiboDao = WeiboInfoDao.getInstance();
					weiboDao.insertWeiboList(curPageLst);
//					curPageLst.clear();
//					curPageLst = null;
				}
				
				
			}
			catch (HttpException e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
				return wbList;
			} 
			catch (IOException e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
				return wbList;
			} 
			catch (InterruptedException e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			finally
			{
				if(!method1.isAborted())
					method1.abort();
				if(!method2.isAborted())
					method2.abort();
				if(!method3.isAborted())
					method3.abort();
			}
			
			pageCount++;
		}
		
		return wbList;
	}
	
	public List<EventBean> downloadEventList(int maxPageCount)
	{
		if(!isLogin)
		{
			//System.out.println("对不起，您还没有登录成功，无法下载微博！");
			init();
		}
		if(!isLogin)
			return null;
		
		List<EventBean> eventList = new ArrayList<EventBean>();
		for(int i = 1; i <= maxPageCount; i++)
		{
			String pageUrl = "http://d.weibo.com/100803?cfs=&Pl_Discover_Pt6Rank__5_filter=hothtlist_type%3D1#_0";
					//String.format("http://d.weibo.com/100803?cfs=&Pl_Discover_Pt6Rank__5_filter=hothtlist_type%3D1#_0");
			GetMethod getMethod = new GetMethod(pageUrl);
			try
			{
				loginClient.executeMethod(getMethod);
				if(getMethod.getStatusCode() != HttpStatus.SC_OK)
				{
					System.out.println("请求数据发生错误!");
					continue;
				}
				byte[] bytes = getMethod.getResponseBody();
				String value = new String(bytes, SpiderConfig.getEventEncoding());
				//System.out.println(value);
				eventList.addAll(weiboParser.parseEventList(value));
				
				//释放连接
				getMethod.releaseConnection();
			} catch (HttpException e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (IOException e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			finally
			{
				if(!getMethod.isAborted())
					getMethod.abort();
			}
			
			
			
			
		}
		
		
		
		return eventList;
		
	}
	
	
	
	
	public List<WeiboBean> downloadAUserWeibo(UserBean ub) throws AccountForbiddonException
	{
		if(!isLogin)
		{
			//System.out.println("对不起，您还没有登录成功，无法下载微博！");
			init();
		}

		if(!isLogin)
			return null;
		if(ub == null)
			return null;
		String uid = ub.getUid();
		List<WeiboBean> wbList = new ArrayList<WeiboBean>();
		/**
		 * 添加代码仅下载用户未下载的微博
		 */
		int downloadedCount = WeiboInfoDao.getInstance().getWeiboCountByUid(uid);
		int needDownloadCount = ub.getWeibo() - downloadedCount;
		int pageCount = needDownloadCount / SpiderConfig.getWeiboCountPerpage();
		if(ub.getWeibo() % SpiderConfig.getWeiboCountPerpage() != 0)
			pageCount++;
		String [] requestUrls = new String [3];
		
		System.out.println(String.format("用户[%s][%s]一共有微博[%d]条，已下载[%d]条，下载剩余的[%d]条", ub.getUid(), ub.getNickName(), ub.getWeibo(), downloadedCount, needDownloadCount));
		
		
		//按照页码依次下载微博
		if(pageCount > SpiderConfig.getMaxPageCount())
			pageCount = SpiderConfig.getMaxPageCount();
		for(int i = 1; i <= pageCount; i++)
		{
			requestUrls[0] = String.format("http://weibo.com/aj/mblog/mbloglist?_wv=5&count=50&page=%d&uid=%s", i, uid);
			requestUrls[1] = String.format("http://weibo.com/aj/mblog/mbloglist?_wv=5&count=15&page=%d&uid=%s&pre_page=%d&pagebar=0&feedtype=0", i, uid, i);
			requestUrls[2] = String.format("http://weibo.com/aj/mblog/mbloglist?_wv=5&count=15&page=%d&uid=%s&pre_page=%d&pagebar=1", i, uid, i);
			
			//三次Get方法请求微博数据
			GetMethod method1 = new GetMethod(requestUrls[0]);
			GetMethod method2 = new GetMethod(requestUrls[1]);
			GetMethod method3 = new GetMethod(requestUrls[2]);
			
			
			try
			{
				List<WeiboBean> curPageLst = new ArrayList<WeiboBean>();
				List<WeiboBean> curReqLst = null;
				loginClient.executeMethod(method1);
				String respStr = SpiderUtility.streamToString(method1.getResponseBodyAsStream(), SpiderConfig.getWeiboEncoding());//method1.getResponseBodyAsString();
				method1.releaseConnection();
				//检查帐号是否异常，若异常抛出异常进行处理
				if(!SpiderUtility.checkJsonResponse(respStr))
					throw new AccountForbiddonException();
				
				curReqLst = weiboParser.parseWeiboList(respStr, ub);
				curPageLst.addAll(curReqLst);

				Thread.currentThread();
				Thread.sleep(SpiderConfig.getWaitTime());
				
				//比较日期，若在一个月之前直接跳过
				//if(curReqLst != null && curReqLst.size() > 0)
				//	weiboDate = df.parse(curReqLst.get(curReqLst.size() - 1).getTime());
				//if(weiboDate.before(breakPoint))
				//	break;
				
				
				
				loginClient.executeMethod(method2);
				respStr = SpiderUtility.streamToString(method2.getResponseBodyAsStream(), SpiderConfig.getWeiboEncoding());
				method2.releaseConnection();
				//检查帐号是否异常，若异常抛出异常进行处理
				if(!SpiderUtility.checkJsonResponse(respStr))
					throw new AccountForbiddonException();
				curReqLst = weiboParser.parseWeiboList(respStr, ub);
				curPageLst.addAll(curReqLst);
				
				//dlWbCount += 15;
				//if(dlWbCount >= ub.getWeibo())
				//	break;
				//比较日期，若在一个月之前直接跳过
				/*
				if(curReqLst != null && curReqLst.size() > 0)
					weiboDate = df.parse(curReqLst.get(curReqLst.size() - 1).getTime());
				if(weiboDate.before(breakPoint))
					break;
					*/
				Thread.currentThread();
				Thread.sleep(SpiderConfig.getWaitTime());
				
				loginClient.executeMethod(method3);
				respStr = SpiderUtility.streamToString(method3.getResponseBodyAsStream(), SpiderConfig.getWeiboEncoding());
				method3.releaseConnection();
				//检查帐号是否异常，若异常抛出异常进行处理
				if(!SpiderUtility.checkJsonResponse(respStr))
					throw new AccountForbiddonException();
				curReqLst = weiboParser.parseWeiboList(respStr, ub);
				//curReqLst = weiboParser.parseWeiboList(respStr, ub);
				curPageLst.addAll(curReqLst);
				wbList.addAll(curPageLst);
				Thread.currentThread();
				Thread.sleep(SpiderConfig.getWaitTime());
				
				SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
				String dateValue = df.format(new Date());
				String forwardInfo = String.format("[%s]已下载 %s[%s] 第[%d/%d]页的[%d]条微博信息", dateValue, ub.getNickName(), ub.getUid(), i, pageCount, curPageLst.size());
				System.out.println(forwardInfo);
				
				
				if(isAccessDataBase)
				{
					WeiboInfoDao weiboDao = WeiboInfoDao.getInstance();
					weiboDao.insertWeiboList(curPageLst);
					curPageLst.clear();
				}
				
			}
			catch (HttpException e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			} 
			catch (IOException e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			} 
			catch (InterruptedException e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			finally
			{
				if(!method1.isAborted())
					method1.abort();
				if(!method2.isAborted())
					method2.abort();
				if(!method3.isAborted())
					method3.abort();
			}
			
			
		}
		return wbList;
	}
	
	
	/**
	 * 下载一个用户的微博信息
	 */
	public List<WeiboBean> downloadAUserWeibo(String uid) throws AccountForbiddonException
	{
		UserBean ub = downloadUserInfo(uid);
		return downloadAUserWeibo(ub);
	}
	
	/**
	 * 下载用户信息
	 * @param uid
	 */
	public UserBean downloadUserInfo(String uid) throws AccountForbiddonException
	{

		if(!isLogin)
			return null;
		String userInfoUrl = String.format("http://weibo.com/%s/info", uid);
		String prepareUrl = String.format("http://weibo.com/%s/", uid);
		GetMethod userInfoMethod = new GetMethod(userInfoUrl);
		GetMethod prepareMethod = new GetMethod(prepareUrl);
		try
		{
			loginClient.executeMethod(userInfoMethod);
			Thread.currentThread();
			Thread.sleep(SpiderConfig.getWaitTime());
			if(userInfoMethod.getStatusCode() != HttpStatus.SC_OK)
			{
				System.out.println("下载用户个人信息返回错误码: " + userInfoMethod.getStatusCode());
				return null;
			}
			String infoString = SpiderUtility.streamToString(userInfoMethod.getResponseBodyAsStream(), SpiderConfig.getUserInfoEncoding());//new String(userInfoMethod.getResponseBody(), SpiderConfig.getUserInfoEncoding());
			
			if(!SpiderUtility.checkHtmlResponse(infoString))
			{
				loginClient.executeMethod(prepareMethod);
				//prepareMethod = new GetMethod(prepareUrl);
				SpiderUtility.streamToString(prepareMethod.getResponseBodyAsStream(), SpiderConfig.getUserInfoEncoding());//new String(userInfoMethod.getResponseBody(), SpiderConfig.getUserInfoEncoding());
			}
			if(!SpiderUtility.checkHtmlResponse(infoString))
			{
				throw new AccountForbiddonException();
			}
			
			UserBean ub = weiboParser.parseUserInfo(infoString);
			if(ub == null)
				return null;
			ub.setUid(uid);
			return ub;
		}
		catch (IOException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
			return null;
		}
		catch (InterruptedException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		finally
		{
			userInfoMethod.releaseConnection();
			prepareMethod.releaseConnection();
		}
		return null;
	}
	
	
	public List<String> downloadFamousPersonUid(String tempUrl)
	{
		if(!isLogin)
			return null;
		List<String> uidList = new ArrayList<String>();
		
		String requestUrl = String.format(tempUrl, 1);
		GetMethod method = new GetMethod(requestUrl);
		try
		{
			loginClient.executeMethod(method);
			String responseStr = SpiderUtility.streamToString(method.getResponseBodyAsStream(), SpiderConfig.getWeiboEncoding());
			method.releaseConnection();
			uidList.addAll(weiboParser.parseFamousPersonUids(responseStr));
			int pageCount = weiboParser.parseFamousPersonPageCount(responseStr);
			for(int i = 2; i <= pageCount; i++)
			{
				System.out.println(String.format("正在下载第%d页的名人信息", i));
				requestUrl = String.format(tempUrl, i);
				method = new GetMethod(requestUrl);
				loginClient.executeMethod(method);
				responseStr = SpiderUtility.streamToString(method.getResponseBodyAsStream(), SpiderConfig.getWeiboEncoding());
				
				uidList.addAll(weiboParser.parseFamousPersonUids(responseStr));
				
				
			}
			
		} catch (HttpException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	
		
		
		
		return uidList;
	}
	
	
	/**
	 * 检查帐号是否异常
	 * @return 正常返回true
	 */
	public boolean checkAccount()
	{
		String checkUrl = "http://www.weibo.com";
		if(!isLogin)
			return false;
		GetMethod method = new GetMethod(checkUrl);
		try
		{
			loginClient.executeMethod(method);
			String resultStr = SpiderUtility.streamToString(method.getResponseBodyAsStream(), SpiderConfig.getWeiboEncoding());
			if(resultStr == null)
				return false;
			String title = weiboParser.getHtmlTitle(resultStr);
			if(title.contains("解冻") || title.contains("验证") || title.contains("激活") || title.contains("404错误"))
				return false;
			return true;
		}
		catch (HttpException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		} 
		catch (IOException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		finally
		{
			method.releaseConnection();
		}
		return false;
	}
	
	
	
	
	
	
	
	
	
	
}